package com.bac.rentmap.ad.crawler.support;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;

import com.bac.rentmap.ad.flat.crawler.ParsingHelper;

public abstract class CrawlingSupport<T> {
    private static final Logger log = Logger.getLogger(CrawlingSupport.class
            .getName());
    private Pattern charsetPattern = Pattern.compile("charset=([\\w-]+)");

    protected InputStreamReader getInputStreamReader(String url)
            throws IOException {
        url = trimUrl(url);
        log.info("Retrieving data from '" + url + "'");
        HttpURLConnection connection = createConnection(url);
        try {
            log.fine("Response headers: " + connection.getHeaderFields());
        } catch (RuntimeException e) {
            log.throwing(this.getClass().getCanonicalName(),
                    "getInputStreamReader", new IOException(
                            "Failed to crawl data from " + url, e));
        }
        if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) {
            throw new IOException("Invalid return code "
                    + connection.getResponseCode());
        }
        InputStream inputStream = connection.getInputStream();
        Charset charset = getCharset(connection);
        if (charset != null)
            return new InputStreamReader(inputStream, charset);
        else
            return new InputStreamReader(inputStream);
    }

    String trimUrl(String url){
        int i = url.indexOf('#');
        String res;
        if(i != -1){
            res = url.substring(0, i);
        }
        else{
            res = url;
        }
        return res;
    }

    protected HttpURLConnection createConnection(String url)
            throws MalformedURLException, IOException {
        HttpURLConnection connection = (HttpURLConnection) new URL(url)
                .openConnection();
        connection.setConnectTimeout(10000);
        return connection;
    }

    protected Charset getCharset(HttpURLConnection connection) {
        String contentTypeValue = connection.getHeaderField("content-type");
        if (contentTypeValue == null) {
            log.warning("Can't parse charset from content type header");
            return null;
        }
        String charset = parseCharset(contentTypeValue);
        if (charset == null) {
            log.warning("Can't parse charset from " + contentTypeValue);
            return null;
        }
        log.fine("Using charset: " + charset);
        return Charset.forName(charset);
    }

    protected String getValue(TagNode tagNode, String xpath)
            throws XPatherException {
        String res = evaluateXPath(tagNode, xpath).toString();
        return convertHtml2text(res);
    }

    protected Object evaluateXPath(TagNode tagNode, String xpath)
            throws XPatherException {
        Object[] xpathValue = tagNode.evaluateXPath(xpath);
        if (xpathValue.length != 1) {
            String message = tagNode.getText().toString();
            int maxLength = 100;
            if (message.length() > maxLength) {
                message = message.substring(0, maxLength) + "...";
            }
            IllegalArgumentException throwable = new IllegalArgumentException(
                    "Can't evaluate XPATH " + xpath + " for " + message
                            + ", length is " + xpathValue.length);
            log.throwing(getClass().getName(), "getValue", throwable);
            throw throwable;
        }
        return xpathValue[0];
    }

    protected String convertHtml2text(CharSequence htmlCharSequence) {
        return ParsingHelper.convertHtml2text(htmlCharSequence);
    }

    abstract protected T parseRootTagNode(TagNode rootNode, T object)
            throws XPatherException;

    protected T crawl(String url, T object) throws IOException {
        BufferedReader reader = new BufferedReader(getInputStreamReader(url));
        CleanerProperties cleanerProperties = new CleanerProperties();
        cleanerProperties.setOmitUnknownTags(true);
        HtmlCleaner htmlCleaner = new HtmlCleaner(cleanerProperties);
        TagNode rootNode = htmlCleaner.clean(reader);
        try {
            return parseRootTagNode(rootNode, object);
        } catch (XPatherException e) {
            throw new RuntimeException("Invalid XPATH", e);
        } finally {
            reader.close();
        }
    }

    public String parseCharset(String contentType) {
        Matcher matcher = charsetPattern.matcher(contentType);
        boolean matchFound = matcher.find();
        if (matchFound) {
            return matcher.group(1);
        }
        return null;
    }
}
